#!/usr/bin/python
# -*- coding: UTF-8 -*-
import tushare as ts
import pandas as pd
import numpy as np
import pymongo
from selenium import webdriver;
from bs4 import BeautifulSoup
from config import *
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import re
browser=webdriver.Chrome()
browser.set_window_size(1400,900)
wait=WebDriverWait(browser, 10);
client=pymongo.MongoClient(MONGO_URL);
db=client["stock"]
def select_item(item,css,**kwargs):
    try:
        my=item.select(css)
        result=None
        if my == None:
          return None
        if len(my)==0:
            return None
        dest=my[0]
        if dest== None:
            return None
        if "text" in kwargs.keys():
            result= dest.get_text().strip()
        if "attrs" in kwargs.keys():
            result =dest.attrs[kwargs.get("attrs")]
        if "extract" in kwargs.keys():
            dest.extract()
    except Exception as e:
        print(e)
        result=None
    return result
def saveToMongo(result):
    try:
        if db["2018426"].insert(result):
            print("save result success");
    except:
        print("save mongo error")
def get_cube_detail(url):
    cube_detail={}
    if url==None:
        return cube_detail
    browser.get(url)
    html=browser.page_source
    soup = BeautifulSoup(html, 'lxml')
    stocks=soup.select(".cube-position .stock-name .name")
    if stocks and len(stocks)>0:
        stocks=[ stock.get_text()  for stock in stocks]
    cube_detail["stocks"]=stocks
    return cube_detail



def search():
    url="https://xueqiu.com/p/discover";
    browser.get(url)
    total = wait.until(
        EC.presence_of_element_located((By.CSS_SELECTOR, "#container > div > div > div > div.cube-content > div.cube-cards.fn-clear.cube-action-discover"))
    );
    html=browser.page_source;

    soup = BeautifulSoup(html, 'lxml')
    items=soup.select(".cube-cards .cube-card")
    for item in items:
        cube_name=select_item(item,".cube-name",text="")
        cube_url = "https://xueqiu.com" + select_item(item, ".cube-name", attrs="href")
        cube_rate=select_item(item,".cube-rate .rate .num",text="")
        cube_tags=select_item(item,".cube-tags",text="")
        cube_auhor=select_item(item,".cube-creator .name",text="")
        cube_author_url="https://xueqiu.com"+select_item(item,".cube-creator .name",attrs="href")
        cube_follwer=select_item(item,".cube-follower .num",text="")
        cube_stocks=get_cube_detail(cube_url).get("stocks",[])
        stock={
            "cube_stocks":cube_stocks,
            "cube_name":cube_name,
        "cube_url":cube_url,
            "cube_rate":cube_rate,
            "cube_tags":cube_tags,
            "cube_author":cube_auhor,
            "cube_author_url":cube_author_url,
            "cube_follwer":cube_follwer
        }
        print(stock)
        saveToMongo(stock)

def get_data():
    queryArgs = {}
    projectionFields = {'_id': False}  # 用字典指定
    result = db["2018424"].find(queryArgs,projectionFields)
    return result


def main():
    search()



if __name__ == '__main__':
    main()

